

#评论
"http://coral.qq.com/article/2619799396/comment/v2?callback=_article2619799396commentv2&orinum=10&oriorder=o&pageflag=1&cursor=0&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525159556754"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6397649407540163448&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998353"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6398819155978193520&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998354"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6397638492388120728&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998355"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6398132922005451852&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998356"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6397958460936551280&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998357"
"http://coral.qq.com/article/2623970564/comment/v2?callback=_article2623970564commentv2&orinum=10&oriorder=o&pageflag=1&cursor=6397779936971296565&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_=1525741998358"


"up: 点赞   repnum: 回复数  id：回复数大于0，通过id，寻找回复 userid:用户id    region：ip   time:时间戳   content:评论内容   hasnext:是否有下一页   last：下一页某个接口参数"


"https://coral.qq.com/article/batchcommentnum?targetid=2635246758&callback=_cbSum&_=1525660316438"

"https://coral.qq.com/article/batchcommentnum?targetid=2635246758&callback=_cbSum&_=1525660316438"
import requests
import json
import execjs
import logging
from pyquery import PyQuery as jq
import time
import pandas as pd
url = "http://roll.news.qq.com/interface/roll.php?0.44306087101927627&cata=newsgn&site=news&date=&page=1&mode=1&of=json"
headers = json.loads(r'''{
    "Accept": "*/*",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    "Cookie": "RK=0bkivATLQP; pgv_pvi=3636279296; eas_sid=Q1c5E0b4l2m7c9W7e986b7d698; tvfe_boss_uuid=c770c9b965b339f0; ptcz=5168196bb1e309f87cb2f68fee1054c28f5ca2978bc4d187691444c1bcc5871b; pt2gguin=o0824300438; pgv_pvid=2783642000; o_cookie=824300438; ptag=www_baidu_com|/; pgv_info=ssid=s6974547792; ts_refer=www.baidu.com/link; ts_uid=7544839156; ts_last=news.qq.com/world_index.shtml; ad_play_index=93; roll_mod=1",
    "Host": "roll.news.qq.com",
    "Referer": "http://roll.news.qq.com/",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}''')
headers2 = json.loads(r'''{
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
    "Connection": "keep-alive",
    "Cookie": "RK=0bkivATLQP; pgv_pvi=3636279296; eas_sid=Q1c5E0b4l2m7c9W7e986b7d698; tvfe_boss_uuid=c770c9b965b339f0; ptcz=5168196bb1e309f87cb2f68fee1054c28f5ca2978bc4d187691444c1bcc5871b; pt2gguin=o0824300438; pgv_pvid=2783642000; o_cookie=824300438; ptag=www_baidu_com|/; pgv_info=ssid=s6974547792; ts_refer=www.baidu.com/link; ts_uid=7544839156; ts_last=news.qq.com/world_index.shtml; ad_play_index=93; roll_mod=1",
    "Host": "coral.qq.com",
    "Referer": "http://roll.news.qq.com/",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
}''')

logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# alist = ['newsgn','newsgj','newssh']
datacol = {0:'新闻标签',1:'新闻来源',2:'新闻发布时间',3:'评论数量',4:'新闻标题',5:'用户名',6:'用户IP',7:'发布时间',8:'顶(数量)',9:'评论内容',10:'回复'}
alist = ['newsgj','newssh']

class news():
	def __init__(self):
		self.allList = []
		self.nextList = []
		self.session = requests.Session()
		self.logger = logging.getLogger("__main__")

	def resp_check(self, resp):
		"""响应处理，429,403错误自动切换用户"""
		netcode = resp.status_code
		if netcode == 200:
			return True
		else:
			return False

	#jsonp转化为json
	def loads_jsonp(self,_jsonp):
		try:
			return json.loads("".join("".join(_jsonp.split("(")[1:]).split(")")[:-1]))
		except:
			raise ValueError('Invalid Input')

	#获取评论列表
	def getNews(self):
		for other in alist:
			for index in range(1,2):
				self.logger.info(f"正在爬取第{index}页新闻数据")
				aa = execjs.eval("Math.random()")
				resp = self.session.get(f"http://roll.news.qq.com/interface/roll.php?{aa}&cata={other}&site=news&date=&page={index}&mode=1&of=json", headers=headers)
				if self.resp_check(resp):
					print(resp.text)
					article_info = json.loads(resp.text)['data']['article_info']
					for uls in jq(article_info)('ul').items():
						for lis in uls('li').items():
							times = lis('.t-time').text()
							types = lis('.t-tit').text()
							title = lis('a').text()
							url = lis('a').attr('href')
							self.getInfo(url,types,times,title)
			with open(f'{other}.json','w',encoding='utf-8') as f:
				json.dump(self.allList, f)
			# df = pd.DataFrame(self.allList)
			# df.rename(columns=datacol,inplace=True)
			# df.to_excel(f'{other}.xlsx')



	#获取来源，详情页
	def getInfo(self,urls,types,times,title):
		resp = self.session.get(urls)
		# print(resq.text)
		lists = resp.text.split('cmt_id = ')
		laiyuan = jq(resp.text)('.qq_main > .qq_article > .hd > .qq_bar.clearfix > .a_Info > .a_source > a').text()
		self.logger.info(f"正在爬取详情页数据{laiyuan}")
		aid = ""
		if len(lists) > 1:
			aid = lists[1].split(';')[0]
			if not aid.strip():
				lists2 = resp.text.split('cmt_id = ')
				if len(lists) > 1:
					aid = lists[1].split('\"')[0]
		print(aid)
		commentnum = self.getCommentNum(aid)
		self.nextList = [types,laiyuan,times,commentnum,title]
		self.allList.append(self.nextList)
		if commentnum is not None and int(commentnum) > 0:
			self.getCommentList(aid,0)


	#获取评论内容
	def getCommentList(self,aid,cursor):
		t = time.time()
		times = int(round(t * 1000))
		url = f"http://coral.qq.com/article/{aid}/comment/v2?callback=_article2619799396commentv2&orinum=10&oriorder=o&pageflag=1&cursor={cursor}&scorecursor=0&orirepnum=2&reporder=o&reppageflag=1&source=1&_={times}"
		resp = self.session.get(url,headers=headers2)
		self.logger.info(f"正在爬取{cursor}评论内容数据")
		if self.resp_check(resp):
			myjson = self.loads_jsonp(resp.text)
			cursor = myjson['data']['last']
			oriCommList = myjson['data']['oriCommList']
			repCommList = myjson['data']['repCommList']
			userList = myjson['data']['userList']
			for oricom in oriCommList:
				findtime = oricom['time']
				userid = oricom['userid']
				content = oricom['content']
				up = oricom['up']
				repnum = oricom['repnum']
				ids = oricom['id']
				name = userList[userid]['nick']
				region = userList[userid]['region']
				strs = ""
				if int(repnum) > 0:
					if ids in repCommList:
						self.logger.info(f"回复id：{ids}")
						for comment in repCommList[str(ids)]:
							strs = strs + comment['content'] + "/"
				self.nextList = ['','','','','',name,region,findtime,up,content,strs]
				self.allList.append(self.nextList)
			self.logger.info(f"是否有下一页{myjson['data']['hasnext']}")
			if myjson['data']['hasnext']:
				self.logger.info(f"获取下一页{myjson['data']['last']}")
				self.getCommentList(aid,myjson['data']['last'])



	#获取评论数量
	def getCommentNum(self,aid):
		self.logger.info(f"正在爬取评论数据{aid}")
		t = time.time()
		times = int(round(t * 1000))
		resp = self.session.get(f'https://coral.qq.com/article/batchcommentnum?targetid={aid}&callback=_cbSum&_={times}',headers=headers2)
		if self.resp_check(resp):
			myjson = self.loads_jsonp(resp.text)
			if 'data' in myjson:
				return myjson['data'][0]['commentnum']
		else:
			return ""
if __name__ == '__main__':
	news().getNews()